import json
import time
import requests
import pandas as pd
from tqdm import tqdm
from datetime import datetime


# 终端执行 python fetch_data.py 写出最新数据到本地


def boards_data():
    """
    以 东方财富网-沪深京板块-行业板块 为例，页面地址：https://quote.eastmoney.com/center/boardlist.html#industry_board
    爬取最新的板块交易概况、板块个股交易概况数据集
    """

    # 获取板块列表
    # 目标接口地址
    target_url = "https://4.push2.eastmoney.com/api/qt/clist/get"
    "fs=m:90+t:2+f:!50&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f26,f22,f33,f11,f62,f128,f136,f115,f152,f124,f107,f104,f105,f140,f141,f207,f208,f209,f222"
    # 构造示例请求参数，若失效，请自行抓包更换
    params = {
        "pn": "1",
        "pz": "99999",
        "po": "1",
        "np": "1",
        "fltt": "2",
        "invt": "2",
        "dect": "1",
        "fid": "f3",
        "fs": "m:90+t:2+f:!50",
        "fields": "f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f26,f22,f33,f11,f62,f128,f136,f115,f152,f124,f107,f104,f105,f140,f141,f207,f208,f209,f222",
    }

    # 发送请求
    response = requests.get(
        target_url,
        params=params,
        headers={
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36"
        },
    )

    boards_list_df = pd.DataFrame(response.json()["data"]["diff"])[
        ["f12", "f13", "f14"]
    ]

    # 获取板块最新数据、板块内部个股最新数据
    raw_boards = {}
    progress = tqdm(boards_list_df.itertuples(), total=boards_list_df.shape[0])
    for row in progress:
        progress.set_description(f"正在爬取板块【{row.f14}】")
        # 板块最新数据
        # 目标接口地址
        target_url = "https://push2his.eastmoney.com/api/qt/stock/kline/get"
        # 构造示例请求参数，若失效，请自行抓包更换
        params = {
            "fields1": "f1,f2,f3,f4,f5,f6",
            "fields2": "f51,f52,f53,f54,f55,f56,f57,f58,f59,f60,f61",
            "klt": "101",
            "fqt": "1",
            "end": "20500101",
            "lmt": "1",
            "secid": "{}.{}".format(row.f13, row.f12),
        }
        # 发送请求
        response = requests.get(
            target_url,
            params=params,
            headers={
                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36"
            },
        )
        raw_boards[row.f12] = {
            "板块统计": [row.f14, *response.json()["data"]["klines"][0].split(",")]
        }

        # 板块内部个股最新数据
        # 目标接口地址
        target_url = "https://67.push2.eastmoney.com/api/qt/clist/get"
        # 构造示例请求参数，若失效，请自行抓包更换
        params = {
            "pn": 1,
            "pz": "99999",
            "po": "1",
            "np": "1",
            "fltt": "2",
            "invt": "2",
            "dect": "1",
            "fid": "f3",
            "fs": "b:{}+f:!50".format(row.f12),
            "fields": "f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152,f45",
        }
        # 发送请求
        response = requests.get(
            target_url,
            params=params,
            headers={
                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36"
            },
        )
        raw_boards[row.f12]["板块内部个股"] = response.json()["data"]["diff"]

        time.sleep(0.1)

    # 数据清洗
    # 板块交易概况
    boards_overview = (
        pd.DataFrame(
            [item["板块统计"] for item in raw_boards.values()],
            columns=[
                "板块名称",
                "交易日期",
                "今开",
                "今收",
                "最高",
                "最低",
                "成交量（万）",
                "成交额（亿）",
                "振幅（%）",
                "涨跌幅（%）",
                "涨跌",
                "换手（%）",
            ],
        )
        # 去除部分字段
        .drop(columns=["涨跌"])
        # 数据类型转换
        .astype(
            {
                "今开": "float",
                "今收": "float",
                "最高": "float",
                "最低": "float",
                "成交量（万）": "float",
                "成交额（亿）": "float",
                "振幅（%）": "float",
                "涨跌幅（%）": "float",
                "换手（%）": "float",
            }
        )
        # 部分数据量纲转换
        .assign(
            **{
                "成交量（万）": lambda df: (df["成交量（万）"] / 10000).round(2),
                "成交额（亿）": lambda df: (df["成交额（亿）"] / 100000000).round(2),
            }
        )
    )

    # 板块个股交易概况
    boards_detail = (
        pd.concat(
            [
                pd.DataFrame(item["板块内部个股"])[
                    [
                        "f12",
                        "f14",
                        "f2",
                        "f3",
                        "f4",
                        "f6",
                        "f7",
                        "f15",
                        "f16",
                        "f17",
                        "f18",
                    ]
                ]
                .rename(
                    columns={
                        "f2": "最新价",
                        "f3": "涨跌幅",
                        "f4": "涨跌额",
                        "f6": "成交额（亿）",
                        "f7": "振幅（%）",
                        "f12": "股票代码",
                        "f14": "股票名称",
                        "f15": "最高",
                        "f16": "最低",
                        "f17": "今开",
                        "f18": "昨收",
                    }
                )
                .assign(所属板块=item["板块统计"][0])
                for item in raw_boards.values()
            ],
            ignore_index=True,
        )
        # 替换特定列中的 "-" 为 None
        .replace('-', None)
        # 数据类型转换
        .astype(
            {
                "最新价": "float",
                "涨跌幅": "float",
                "涨跌额": "float",
                "成交额（亿）": "float",
                "振幅（%）": "float",
                "最高": "float",
                "最低": "float",
                "今开": "float",
                "昨收": "float",
            }
        )
        # 部分数据量纲转换
        .assign(
            **{
                "成交额（亿）": lambda df: (df["成交额（亿）"] / 100000000).round(2),
            }
        )
    )

    return boards_overview, boards_detail


if __name__ == "__main__":
    # 写出演示数据到本地
    boards_overview, boards_detail = boards_data()
    boards_overview.to_csv("板块交易概况.csv", index=False)
    boards_detail.to_csv("板块个股交易概况.csv", index=False)
    # 写出数据更新相关信息
    with open("./data_log.json", "w", encoding="utf-8") as f:
        json.dump({"数据更新时间": datetime.now().strftime("%Y-%m-%d %H:%M:%S")}, f)
    print("-" * 100)
    print("板块交易概况、板块个股交易概况已写出到本地！")
